﻿module Keywords

open System
open System.IO
open System.Net
open System.Reflection
open System.Runtime.Serialization
open System.Text.RegularExpressions

module private Helpers =

    /// Removes HTML tags, comments, inline script code and CSS definitions.
    let extractText html =
        let pattern = "(?is)<!--.*?--\s*>|<script.*?</script>|<style.*?</style>"
        let html' = Regex(pattern, RegexOptions.Compiled).Replace(html, " ")
        let tagsFree = Regex("(?i)<[^>]*>", RegexOptions.Compiled).Replace(html', "  ") |> WebUtility.HtmlDecode
        let tagsFree' = Regex.Replace(tagsFree, "\s{3,}", "  ")
        tagsFree'.ToLower()

    /// Detects the language of a string.
    let detectLang (str : string) =
        try
            let appId = "Bing app ID"
            let tooLong = str.Length > 100
            let text =
                match tooLong with
                | true -> str.Substring(0, 100)
                | false -> str  
            let url = "http://api.microsofttranslator.com/v2/Http.svc/Detect?appId=" + appId + "&text=" + text
            use stream = WebRequest.Create(url).GetResponse().GetResponseStream()
            let dcs = new DataContractSerializer(text.GetType())
            dcs.ReadObject(stream) |> string
        with
        | _ -> "en"

    /// Loads the appropriate stop words list for the given language.
    let getStopWords lang =
        let load name = 
            let file = Assembly.GetExecutingAssembly().GetManifestResourceNames() |> Array.find (fun x -> x = name)
            use stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(file)
            use reader = new StreamReader(stream)
            [ 
                while not reader.EndOfStream do
                    yield reader.ReadLine()
            ]

        match lang with
        | "en" -> load "enStpWords.txt"
        | _ -> load "frStpWords.txt"

    /// Removes stop words from a string.
    let rec remStpWords input lst =
        match lst with
        | h :: t -> remStpWords (Regex("\\b" + h + "\\b").Replace(input, "0")) t
        | [] -> input

    /// Extracts keywords.
    let extractKeywords str = 
        [
            for x in Regex("[^0-9\W]+", RegexOptions.Compiled).Matches(str) do
                yield x.Value
        ]

    /// Calculates the number of occurrences and the density of a keyword.
    let index str len len' lst =
        [
            for x in lst do
                let count = Regex(@"\b" + Regex.Escape(x) + @"\b").Matches(str).Count
                if count > 1 then
                    yield (x, count, (float count / len * 100. * len'))
        ]
        |> List.sortBy (fun (_, x, _) -> x)
        |> List.rev

    /// Helper for extracting 2 and 3 words keyword combinations.
    let matches str lst = 
        [
            for x in lst do
                let a, _, _ = x
                let matches = Regex("\\b" + a + "\\ [^0-9\W]+\\b").Matches(str)
                for y in matches do
                    yield y.Value
        ]
        |> Seq.distinct

open Helpers

/// Extracts keywords from HTML.
let indexHtml html =                       
    let html' = extractText html
    let lang = detectLang html'
    let stopWords = getStopWords lang
    let html'' = remStpWords html' stopWords           
    let keywords = extractKeywords html''
    let keywords' = keywords |> Seq.distinct
    let length = keywords.Length |> float

    // Single word.
    let oneWord = index html'' length 1. keywords' |> List.map (fun (x, y, z) -> x, y, Math.Round(z, 2))

    // Two words.
    let twoWords = oneWord |> matches html''
                           |> index html'' length 2.0 
                           |> List.map (fun (x, y, z) -> x, y, Math.Round(z, 2))
    // Three words.
    let threeWords = twoWords |> matches html''
                              |> index html'' length 3.0
                              |> List.map (fun (x, y, z) -> x, y, Math.Round(z, 2))
    [oneWord; twoWords; threeWords]